package com.ctbri.manage.bydeequ.example
import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions._
import org.apache.spark.sql.expressions._

/**
 * @author songyunlong
 * @createTime 2023/6/12 09:26
 * @description
 */
object Demo extends App {
    val spark = SparkSession.builder()
        .master("local")
        .appName("test")
        .config("spark.ui.enabled", "false")
        .getOrCreate()
    import spark.implicits._
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    val rdd = spark.sparkContext.parallelize(Seq(
        Item(1, "Thingy A", "awesome thing.", "high", 0),
        Item(2, "Thingy B", "available at http://thingb.com", null, 0),
        Item(3, null, null, "low", 5),
        Item(4, "Thingy D", "checkout https://thingd.ca", "low", 10),
        Item(5, "Thingy E", null, "high", 12)))
    val data = spark.createDataFrame(rdd)
    data.show()
    val data1 = data.select($"*" +: Seq("numViews", "id").map(c=>sum(c).over(Window.orderBy("id")).alias(s"cum$c")): _*)
    data1.show()
    val data2 = Seq("numViews", "id").foldLeft(data)((data, c)=>
        data.withColumn(s"cum$c", sum(c).over(Window.orderBy("id")).alias(s"cum$c")))
    data2.show()
    data.withColumn("cumd", $"id"+1).show()
}
